import scrapy

class DmozSpider(scrapy.Spider):
    name = "all"
    start_urls = []
    
    def __init__(self):
        DmozSpider.start_urls.append('http://www.dingniugu.com/newdata/stockcode.js')
            
    def parse(self, response):
        filename = response.url.split("/")[-1]

        doc = response.body
        pos1 = doc.find('(')
        pos2 = doc.rfind(')')
        content = doc[pos1:pos2+1]
        print content
        
        l1 = tuple(eval(content))
        
        l2 = {
              'sh600001', 'sh600003', 'sh600087', 'sh600102', 'sh600253', 'sh600263', 'sh600357', 
              'sh600553', 'sh600591', 'sh600607', 'sh600627', 'sh600631', 'sh600786', 'sh600832',
              'sh600840', 'sh600842', 'sh600849', 'sh600991', 'sh601268', 'sh601299', 'sz000024', 
              'sz000033', 'sz000515', 'sz000522', 'sz000527', 'sz000562', 'sz000569', 'sz000578',
              'sz000594', 'sz000602', 'sz002999', 'sz300186', 'sh600656',
            }
        
        with open('allstocksname.py', 'w') as f:
            f.writelines("#coding=utf-8\n\n")
            f.writelines("g_StockNameDict = {\n")
        
            for item in l1:
                try:
                    if item[0][0] != '0' and item[0][0] != '3' and item[0][0] != '6':
                        continue
                        
                    if item[0] == "399001":
                        continue
                    
                    if item[0][0] == '6':
                        suffix = "sh"
                    else:
                        suffix = "sz"
                        
                    str = item[1]
                    id = suffix + item[0]
                    
                    if id in l2:
                        continue
                    
                    print type(str), str
                    f.write('\t\''+id+'\' : '+'\''+str.decode('gbk').encode('utf8')+'\',\n')
                except Exception as e:
                    print e
        
            f.writelines("}\n")        

        
